From 4cca16ccfc3441bafeb3ee10831c42878a1df153 Mon Sep 17 00:00:00 2001 From: "kaf24@firebug.cl.cam.ac.uk" Date: Tue, 12 Jul 2005 10:29:46 +0000 Subject: [PATCH] Fix dma_map_single to work correctly with multi-page buffers. Signed-off-by: Keir Fraser --- .../arch/xen/i386/kernel/pci-dma.c | 128 +++++++++++++++++ .../arch/xen/x86_64/kernel/pci-dma.c | 131 ++++++++++++++++++ .../include/asm-xen/asm-i386/dma-mapping.h | 32 ++--- .../include/asm-xen/asm-x86_64/dma-mapping.h | 59 +------- 4 files changed, 275 insertions(+), 75 deletions(-) diff --git a/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c b/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c index efd0dab89b..1fdda04395 100644 --- a/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c +++ b/linux-2.6-xen-sparse/arch/xen/i386/kernel/pci-dma.c @@ -152,3 +152,131 @@ void *dma_mark_declared_memory_occupied(struct device *dev, return mem->virt_base + (pos << PAGE_SHIFT); } EXPORT_SYMBOL(dma_mark_declared_memory_occupied); + +static LIST_HEAD(dma_map_head); +static DEFINE_SPINLOCK(dma_map_lock); +struct dma_map_entry { + struct list_head list; + dma_addr_t dma; + char *bounce, *host; + size_t size; +}; +#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d))) + +dma_addr_t +dma_map_single(struct device *dev, void *ptr, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + void *bnc; + dma_addr_t dma; + unsigned long flags; + + BUG_ON(direction == DMA_NONE); + + /* + * Even if size is sub-page, the buffer may still straddle a page + * boundary. Take into account buffer start offset. All other calls are + * conservative and always search the dma_map list if it's non-empty. + */ + if ((((unsigned int)ptr & ~PAGE_MASK) + size) <= PAGE_SIZE) { + dma = virt_to_bus(ptr); + } else { + BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, 0)) == NULL); + BUG_ON((ent = kmalloc(sizeof(*ent), GFP_KERNEL)) == NULL); + if (direction != DMA_FROM_DEVICE) + memcpy(bnc, ptr, size); + ent->dma = dma; + ent->bounce = bnc; + ent->host = ptr; + ent->size = size; + spin_lock_irqsave(&dma_map_lock, flags); + list_add(&ent->list, &dma_map_head); + spin_unlock_irqrestore(&dma_map_lock, flags); + } + + flush_write_buffers(); + return dma; +} +EXPORT_SYMBOL(dma_map_single); + +void +dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + unsigned long flags; + + BUG_ON(direction == DMA_NONE); + + /* Fast-path check: are there any multi-page DMA mappings? */ + if (!list_empty(&dma_map_head)) { + spin_lock_irqsave(&dma_map_lock, flags); + list_for_each_entry ( ent, &dma_map_head, list ) { + if (DMA_MAP_MATCHES(ent, dma_addr)) { + list_del(&ent->list); + break; + } + } + spin_unlock_irqrestore(&dma_map_lock, flags); + if (&ent->list != &dma_map_head) { + BUG_ON(dma_addr != ent->dma); + BUG_ON(size != ent->size); + if (direction != DMA_TO_DEVICE) + memcpy(ent->host, ent->bounce, size); + dma_free_coherent(dev, size, ent->bounce, ent->dma); + kfree(ent); + } + } +} +EXPORT_SYMBOL(dma_unmap_single); + +void +dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + unsigned long flags, off; + + /* Fast-path check: are there any multi-page DMA mappings? */ + if (!list_empty(&dma_map_head)) { + spin_lock_irqsave(&dma_map_lock, flags); + list_for_each_entry ( ent, &dma_map_head, list ) + if (DMA_MAP_MATCHES(ent, dma_handle)) + break; + spin_unlock_irqrestore(&dma_map_lock, flags); + if (&ent->list != &dma_map_head) { + off = dma_handle - ent->dma; + BUG_ON((off + size) > ent->size); + if (direction != DMA_TO_DEVICE) + memcpy(ent->host+off, ent->bounce+off, size); + } + } +} +EXPORT_SYMBOL(dma_sync_single_for_cpu); + +void +dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + unsigned long flags, off; + + /* Fast-path check: are there any multi-page DMA mappings? */ + if (!list_empty(&dma_map_head)) { + spin_lock_irqsave(&dma_map_lock, flags); + list_for_each_entry ( ent, &dma_map_head, list ) + if (DMA_MAP_MATCHES(ent, dma_handle)) + break; + spin_unlock_irqrestore(&dma_map_lock, flags); + if (&ent->list != &dma_map_head) { + off = dma_handle - ent->dma; + BUG_ON((off + size) > ent->size); + if (direction != DMA_FROM_DEVICE) + memcpy(ent->bounce+off, ent->host+off, size); + } + } + + flush_write_buffers(); +} +EXPORT_SYMBOL(dma_sync_single_for_device); diff --git a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c index 4f58329fae..e8f80155a2 100644 --- a/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c +++ b/linux-2.6-xen-sparse/arch/xen/x86_64/kernel/pci-dma.c @@ -203,3 +203,134 @@ void *dma_mark_declared_memory_occupied(struct device *dev, } EXPORT_SYMBOL(dma_mark_declared_memory_occupied); #endif + +static LIST_HEAD(dma_map_head); +static DEFINE_SPINLOCK(dma_map_lock); +struct dma_map_entry { + struct list_head list; + dma_addr_t dma; + char *bounce, *host; + size_t size; +}; +#define DMA_MAP_MATCHES(e,d) (((e)->dma<=(d)) && (((e)->dma+(e)->size)>(d))) + +dma_addr_t +dma_map_single(struct device *dev, void *ptr, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + void *bnc; + dma_addr_t dma; + unsigned long flags; + + if (direction == DMA_NONE) + out_of_line_bug(); + + /* + * Even if size is sub-page, the buffer may still straddle a page + * boundary. Take into account buffer start offset. All other calls are + * conservative and always search the dma_map list if it's non-empty. + */ + if ((((unsigned int)ptr & ~PAGE_MASK) + size) <= PAGE_SIZE) { + dma = virt_to_bus(ptr); + } else { + BUG_ON((bnc = dma_alloc_coherent(dev, size, &dma, 0)) == NULL); + BUG_ON((ent = kmalloc(sizeof(*ent), GFP_KERNEL)) == NULL); + if (direction != DMA_FROM_DEVICE) + memcpy(bnc, ptr, size); + ent->dma = dma; + ent->bounce = bnc; + ent->host = ptr; + ent->size = size; + spin_lock_irqsave(&dma_map_lock, flags); + list_add(&ent->list, &dma_map_head); + spin_unlock_irqrestore(&dma_map_lock, flags); + } + + if ((dma+size) & ~*hwdev->dma_mask) + out_of_line_bug(); + return dma; +} +EXPORT_SYMBOL(dma_map_single); + +void +dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + unsigned long flags; + + if (direction == DMA_NONE) + out_of_line_bug(); + + /* Fast-path check: are there any multi-page DMA mappings? */ + if (!list_empty(&dma_map_head)) { + spin_lock_irqsave(&dma_map_lock, flags); + list_for_each_entry ( ent, &dma_map_head, list ) { + if (DMA_MAP_MATCHES(ent, dma_addr)) { + list_del(&ent->list); + break; + } + } + spin_unlock_irqrestore(&dma_map_lock, flags); + if (&ent->list != &dma_map_head) { + BUG_ON(dma_addr != ent->dma); + BUG_ON(size != ent->size); + if (direction != DMA_TO_DEVICE) + memcpy(ent->host, ent->bounce, size); + dma_free_coherent(dev, size, ent->bounce, ent->dma); + kfree(ent); + } + } +} +EXPORT_SYMBOL(dma_unmap_single); + +void +dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + unsigned long flags, off; + + /* Fast-path check: are there any multi-page DMA mappings? */ + if (!list_empty(&dma_map_head)) { + spin_lock_irqsave(&dma_map_lock, flags); + list_for_each_entry ( ent, &dma_map_head, list ) + if (DMA_MAP_MATCHES(ent, dma_handle)) + break; + spin_unlock_irqrestore(&dma_map_lock, flags); + if (&ent->list != &dma_map_head) { + off = dma_handle - ent->dma; + BUG_ON((off + size) > ent->size); + if (direction != DMA_TO_DEVICE) + memcpy(ent->host+off, ent->bounce+off, size); + } + } +} +EXPORT_SYMBOL(dma_sync_single_for_cpu); + +void +dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, + enum dma_data_direction direction) +{ + struct dma_map_entry *ent; + unsigned long flags, off; + + /* Fast-path check: are there any multi-page DMA mappings? */ + if (!list_empty(&dma_map_head)) { + spin_lock_irqsave(&dma_map_lock, flags); + list_for_each_entry ( ent, &dma_map_head, list ) + if (DMA_MAP_MATCHES(ent, dma_handle)) + break; + spin_unlock_irqrestore(&dma_map_lock, flags); + if (&ent->list != &dma_map_head) { + off = dma_handle - ent->dma; + BUG_ON((off + size) > ent->size); + if (direction != DMA_FROM_DEVICE) + memcpy(ent->bounce+off, ent->host+off, size); + } + } + + flush_write_buffers(); +} +EXPORT_SYMBOL(dma_sync_single_for_device); diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h index 41ac456d12..2dc373e733 100644 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-i386/dma-mapping.h @@ -16,21 +16,13 @@ void *dma_alloc_coherent(struct device *dev, size_t size, void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -static inline dma_addr_t +extern dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(direction == DMA_NONE); - flush_write_buffers(); - return virt_to_bus(ptr); -} + enum dma_data_direction direction); -static inline void +extern void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size, - enum dma_data_direction direction) -{ - BUG_ON(direction == DMA_NONE); -} + enum dma_data_direction direction); static inline int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, @@ -73,24 +65,20 @@ dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries, BUG_ON(direction == DMA_NONE); } -static inline void +extern void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ -} + enum dma_data_direction direction); -static inline void +extern void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, - enum dma_data_direction direction) -{ - flush_write_buffers(); -} + enum dma_data_direction direction); static inline void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction direction) { + dma_sync_single_for_cpu(dev, dma_handle+offset, size, direction); } static inline void @@ -98,7 +86,7 @@ dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle, unsigned long offset, size_t size, enum dma_data_direction direction) { - flush_write_buffers(); + dma_sync_single_for_device(dev, dma_handle+offset, size, direction); } static inline void diff --git a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h index 8d4e666cf4..295183bbaf 100644 --- a/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h +++ b/linux-2.6-xen-sparse/include/asm-xen/asm-x86_64/dma-mapping.h @@ -21,68 +21,21 @@ void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle void dma_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_handle); -#ifdef CONFIG_GART_IOMMU - extern dma_addr_t dma_map_single(struct device *hwdev, void *ptr, size_t size, int direction); extern void dma_unmap_single(struct device *dev, dma_addr_t addr,size_t size, int direction); -#else - -/* No IOMMU */ - -static inline dma_addr_t dma_map_single(struct device *hwdev, void *ptr, - size_t size, int direction) -{ - dma_addr_t addr; - - if (direction == DMA_NONE) - out_of_line_bug(); - addr = virt_to_machine(ptr); - - if ((addr+size) & ~*hwdev->dma_mask) - out_of_line_bug(); - return addr; -} - -static inline void dma_unmap_single(struct device *hwdev, dma_addr_t dma_addr, - size_t size, int direction) -{ - if (direction == DMA_NONE) - out_of_line_bug(); - /* Nothing to do */ -} -#endif - #define dma_map_page(dev,page,offset,size,dir) \ dma_map_single((dev), page_address(page)+(offset), (size), (dir)) -static inline void dma_sync_single_for_cpu(struct device *hwdev, - dma_addr_t dma_handle, - size_t size, int direction) -{ - if (direction == DMA_NONE) - out_of_line_bug(); - - if (swiotlb) - return swiotlb_sync_single_for_cpu(hwdev,dma_handle,size,direction); - - flush_write_buffers(); -} - -static inline void dma_sync_single_for_device(struct device *hwdev, - dma_addr_t dma_handle, - size_t size, int direction) -{ - if (direction == DMA_NONE) - out_of_line_bug(); - - if (swiotlb) - return swiotlb_sync_single_for_device(hwdev,dma_handle,size,direction); +extern void +dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, size_t size, + int direction); - flush_write_buffers(); -} +extern void +dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle, size_t size, + int direction); static inline void dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, -- 2.30.2